﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;
namespace Crawl {
    public class Crawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private Uri baseurl = new Uri("http://www.cnblogs.com/dstang2000/");
        public Form1 r = null;
        public delegate void event_handler(string sender);
        public event event_handler tmp;
        public static void Main(string startUrl, Form1 r=null)
        {
            Crawler myCrawler = new Crawler();
            if (startUrl.Length<=1)startUrl = "http://www.cnblogs.com/dstang2000/";
            myCrawler.r = r;
            myCrawler.tmp += myCrawler.r.update_Field;
            myCrawler.urls.Add(startUrl, false);//加入初抬page
            new Thread(myCrawler.Crawl).Start();//开始爬
        }
        private void Crawl()
        {
            //Console.WriteLine("开始爬行了...");
            if (r != null) updateField("开始爬行了...\n");
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 10) break;
                if (r != null) updateField("爬行" + current + "页面！\n");
                string html = DownLoad(current);
                urls[current] = true;
                count++;
                Parse(html);
            }
            if (r != null) updateField("爬行结束\n");
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString()+".html";
               // updateField(fileName);
                File.WriteAllText(fileName, html, Encoding.UTF8);
                baseurl = new Uri(url);
                return html;
            }
            catch (Exception ex)
            {
                updateField("爬取" + url + "错误：");
                updateField(ex.Message+"\n");
                //throw;
                return "";
            }
        }
        public void Parse(string html)
        {
            string strRef = "<a.+?href=\"(.+?)\".*>";
            if (!html.Contains("html")) { return; }//判断是否包含html关键字来决定是否为html文件，是则加入爬取队列，不是直接不入队。
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Groups[1].Value;
                //strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (strRef.Contains("./"))
                {
                    strRef = new Uri(baseurl, strRef).ToString();//转换相对地址为绝对地址，采用内置的Uri对象进行转换
                }
                if (strRef.Length == 0) continue;
                if ((urls[strRef] == null) && strRef.Contains("http")) { urls[strRef] = false;
                    //updateField(strRef + "\n");
                }
            }
        }


        public void updateField(string s)
        {
            tmp(s);
        }
    }
    
}